import logging
import os
import re
import threading

import requests

logging.basicConfig(
    filename="baidu_pics_download_6-29.log",
    level=logging.INFO
)
num_every = 1000  # 每个关键词下载的数量
keywords = ["日漫男", "日漫女", "日漫老人男", "日漫老人女", "日漫大叔", "日漫阿姨"] # 关键词数组形式
result_dir = ""  # 下载位置


def download_by_keyword(keyword):
    """
    根据一个关键词进行图片下载
    :param keyword:
    :return:
    """
    r1 = re.compile('"objURL":"(.*?)",')
    index_pre = 0
    urls = [f'http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word={keyword}&pn={id}' for id in
            range(1, num_every, 60)]
    outdir = keyword
    if not os.path.exists(outdir):
        os.mkdir(outdir)
    for url in urls:
        logging.info(f"url{url}")
        print(url)
        res = requests.get(url)
        pic_urls = r1.findall(res.text, re.S)
        index_in_true = 0  # 在该次循环中真实下载的图片数
        index_in_whole = 0
        for pic_urls in pic_urls:
            try:
                with open(os.path.join(result_dir, outdir, f"{index_pre}_{index_in_true}_{index_in_whole}.jpg"),
                          'wb') as fp:
                    res = requests.get(pic_urls, timeout=7, stream=True)
                    fp.write(res.content)
                index_in_true += 1
            except Exception as e:
                logging.error(f"{keyword}:{index_in_true}/{index_in_whole}-{index_pre} error")
            finally:
                index_in_whole += 1
                index_pre += 1
                logging.info(f"{keyword}:{index_in_true}/{index_in_whole}-{index_pre}")


if __name__ == '__main__':
    # 多线程执行
    ts = []
    for keyword in keywords:
        t = threading.Thread(target=download_by_keyword, args=(keyword,))
        t.start()
        print(f"{keyword} Thread starting")
        logging.info(f"{keyword} Thread starting")
        ts.append(t)
    for t in ts:
        t.join()
    print("程序结束")
